Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Merge branch 'master' into omniauth

Conflicts:
Gemfile.lock

Dominik Sander 10 years ago
parent
fe192e8ca5 0f1b30bd9c
commit
00663c153c
21 changed files with 332 additions and 91 deletions
Show Diff Stats
  1. 9 5
      .env.example
  2. 56 0
      Dockerfile
  3. 15 8
      Gemfile
  4. 34 34
      Gemfile.lock
  5. 4 0
      app/assets/javascripts/application.js.coffee.erb
  6. 6 0
      app/assets/stylesheets/application.css.scss.erb
  7. 15 4
      app/concerns/web_request_concern.rb
  8. 28 14
      app/helpers/application_helper.rb
  9. 7 0
      app/helpers/markdown_helper.rb
  10. 2 0
      app/models/agent.rb
  11. 1 1
      app/models/agents/hipchat_agent.rb
  12. 40 7
      app/models/agents/website_agent.rb
  13. 4 4
      app/views/agents/show.html.erb
  14. 7 1
      app/views/layouts/_navigation.html.erb
  15. 1 1
      app/views/scenario_imports/_step_two.html.erb
  16. 1 1
      app/views/scenarios/show.html.erb
  17. 8 8
      lib/twitter_stream.rb
  18. 14 0
      spec/helpers/markdown_helper_spec.rb
  19. 2 2
      spec/models/agents/hipchat_agent_spec.rb
  20. 52 0
      spec/models/agents/website_agent_spec.rb
  21. 26 1
      spec/support/shared_examples/web_request_concern.rb

+ 9 - 5
.env.example
View File 

@@ -99,13 +99,17 @@ AWS_SANDBOX=false
99 99 
 #   Various Settings   #
100 100 
 ########################
101 101
102 
-# Specify the HTTP backend library for Faraday, used in WebsiteAgent.
103 
-# You can change this depending on the performance and stability you
104 
-# need for your service.  Any choice other than "typhoeus",
105 
-# "net_http", or "em_http" should require you to bundle a corresponding
106 
-# gem via Gemfile.
102 
+# Specify the HTTP backend library for Faraday, commonly used by
103 
+# WebsiteAgent, RssAgent and PostAgent.  You can change this depending
104 
+# on the performance and stability you need for your service.  Any
105 
+# choice other than "typhoeus", "net_http", or "em_http" should
106 
+# require you to bundle a corresponding gem via Gemfile.
107 107 
 FARADAY_HTTP_BACKEND=typhoeus
108 108
109 
+# Specify the default User-Agent header value for HTTP requests made
110 
+# by Agents that allow overriding the User-Agent header value.
111 
+DEFAULT_HTTP_USER_AGENT="Huginn - https://github.com/cantino/huginn"
112 
+
109 113 
 # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)]
110 114 
 # You should not allow this on a shared Huginn box because it is not secure.
111 115 
 ALLOW_JSONPATH_EVAL=false

+ 56 - 0
Dockerfile
View File 

@@ -0,0 +1,56 @@
1 
+FROM ubuntu
2 
+# MAINTAINER Someone <someone@example.com>
3 
+
4 
+# Update package list
5 
+RUN apt-get update
6 
+
7 
+# Set environmental variables
8 
+ENV HOME /root
9 
+ENV RBENV_ROOT $HOME/.rbenv
10 
+ENV RUBY_VERSION 1.9.3-p545
11 
+ENV RUBYGEMS_VERSION 2.2.2
12 
+ENV PATH $HOME/.rbenv/shims:$HOME/.rbenv/bin:$RBENV_ROOT/versions/$RUBY_VERSION/bin:$PATH
13 
+
14 
+# Install OS packages
15 
+RUN apt-get install -y build-essential curl zlib1g-dev libreadline-dev libssl-dev libcurl4-openssl-dev git libmysqlclient-dev
16 
+
17 
+RUN git clone https://github.com/sstephenson/rbenv.git $HOME/.rbenv
18 
+RUN git clone https://github.com/sstephenson/ruby-build.git $HOME/.rbenv/plugins/ruby-build
19 
+
20 
+# install & set global ruby version
21 
+RUN rbenv install $RUBY_VERSION
22 
+RUN rbenv global $RUBY_VERSION
23 
+
24 
+WORKDIR /usr/local/src
25 
+
26 
+RUN curl -O http://production.cf.rubygems.org/rubygems/rubygems-$RUBYGEMS_VERSION.tgz
27 
+RUN tar -xvf rubygems-$RUBYGEMS_VERSION.tgz
28 
+RUN cd rubygems-$RUBYGEMS_VERSION ; ruby setup.rb
29 
+
30 
+RUN gem install bundle
31 
+
32 
+RUN mkdir huginn
33 
+WORKDIR huginn
34 
+
35 
+# Add Gemfiles and run bundle ahead of time
36 
+# This way bundle does not have to rerun unless the Gemfile changes
37 
+# It drastically speeds up rebuilds
38 
+ADD Gemfile /usr/local/src/huginn/
39 
+ADD Procfile /usr/local/src/huginn/
40 
+ADD Gemfile.lock /usr/local/src/huginn/
41 
+RUN bundle
42 
+
43 
+# Now add the rest of the source
44 
+ADD . /usr/local/src/huginn/
45 
+RUN rm -rf /usr/local/src/huginn/.env
46 
+
47 
+# Add the environmental variables this way so that the -e option can override them
48 
+ENV DATABASE_HOST db
49 
+ENV DATABASE_NAME huginn
50 
+ENV DATABASE_USERNAME huginn
51 
+
52 
+# Expose the Rails port to the rest of the world
53 
+EXPOSE 3000
54 
+
55 
+# Default command - optimized for upgradability
56 
+CMD ["foreman", "start"]

+ 15 - 8
Gemfile
View File 

@@ -11,17 +11,25 @@ gem 'bundler', '>= 1.5.0'
11 11
12 12 
 gem 'protected_attributes', '~>1.0.8'
13 13
14 
-gem 'rails' , '4.1.4'
14 
+gem 'rails' , '4.1.5'
15 15
16 16 
 case RUBY_PLATFORM
17 
-when /freebsd/
18 
-  # Seems FreeBSD's zoneinfo is not exactly what tzinfo expects
19 
-  gem 'tzinfo-data'
20 
-else
21 
-  # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
22 
-  gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
17 
+when /freebsd|netbsd|openbsd/
18 
+  # ffi (required by typhoeus via ethon) merged fixes for bugs fatal
19 
+  # on these platforms after 1.9.3; no following release as yet.
20 
+  gem 'ffi', github: 'ffi/ffi', branch: 'master'
21 
+
22 
+  # tzinfo 1.2.0 has added support for reading zoneinfo on these
23 
+  # platforms.
24 
+  gem 'tzinfo', '>= 1.2.0'
25 
+when /solaris/
26 
+  # ditto
27 
+  gem 'tzinfo', '>= 1.2.0'
23 28 
 end
24 29
30 
+# Windows does not have zoneinfo files, so bundle the tzinfo-data gem.
31 
+gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
32 
+
25 33 
 gem 'mysql2', '~> 0.3.16'
26 34 
 gem 'devise', '~> 3.2.4'
27 35 
 gem 'kaminari', '~> 0.16.1'
 
@@ -120,4 +128,3 @@ else
120 128 
   gem 'unicorn', platform: :ruby_18
121 129 
   gem 'rails_12factor', platform: :ruby_18
122 130 
 end
123 
-

+ 34 - 34
Gemfile.lock
View File 

@@ -12,27 +12,27 @@ GEM
12 12 
   remote: https://rubygems.org/
13 13 
   specs:
14 14 
     ace-rails-ap (2.0.1)
15 
-    actionmailer (4.1.4)
16 
-      actionpack (= 4.1.4)
17 
-      actionview (= 4.1.4)
15 
+    actionmailer (4.1.5)
16 
+      actionpack (= 4.1.5)
17 
+      actionview (= 4.1.5)
18 18 
       mail (~> 2.5.4)
19 
-    actionpack (4.1.4)
20 
-      actionview (= 4.1.4)
21 
-      activesupport (= 4.1.4)
19 
+    actionpack (4.1.5)
20 
+      actionview (= 4.1.5)
21 
+      activesupport (= 4.1.5)
22 22 
       rack (~> 1.5.2)
23 23 
       rack-test (~> 0.6.2)
24 
-    actionview (4.1.4)
25 
-      activesupport (= 4.1.4)
24 
+    actionview (4.1.5)
25 
+      activesupport (= 4.1.5)
26 26 
       builder (~> 3.1)
27 27 
       erubis (~> 2.7.0)
28 
-    activemodel (4.1.4)
29 
-      activesupport (= 4.1.4)
28 
+    activemodel (4.1.5)
29 
+      activesupport (= 4.1.5)
30 30 
       builder (~> 3.1)
31 
-    activerecord (4.1.4)
32 
-      activemodel (= 4.1.4)
33 
-      activesupport (= 4.1.4)
31 
+    activerecord (4.1.5)
32 
+      activemodel (= 4.1.5)
33 
+      activesupport (= 4.1.5)
34 34 
       arel (~> 5.0.0)
35 
-    activesupport (4.1.4)
35 
+    activesupport (4.1.5)
36 36 
       i18n (~> 0.6, >= 0.6.9)
37 37 
       json (~> 1.7, >= 1.7.7)
38 38 
       minitest (~> 5.1)
 
@@ -65,7 +65,7 @@ GEM
65 65 
       execjs
66 66 
     coffee-script-source (1.7.1)
67 67 
     cookiejar (0.3.2)
68 
-    coveralls (0.7.0)
68 
+    coveralls (0.7.1)
69 69 
       multi_json (~> 1.3)
70 70 
       rest-client
71 71 
       simplecov (>= 0.7)
 
@@ -77,7 +77,7 @@ GEM
77 77 
     debug_inspector (0.0.2)
78 78 
     delayed_job (4.0.2)
79 79 
       activesupport (>= 3.0, < 4.2)
80 
-    delayed_job_active_record (4.0.1)
80 
+    delayed_job_active_record (4.0.2)
81 81 
       activerecord (>= 3.0, < 4.2)
82 82 
       delayed_job (>= 3.0, < 4.1)
83 83 
     delorean (2.1.0)
 
@@ -223,7 +223,7 @@ GEM
223 223 
     polyglot (0.3.5)
224 224 
     protected_attributes (1.0.8)
225 225 
       activemodel (>= 4.0.1, < 5.0)
226 
-    pry (0.10.0)
226 
+    pry (0.10.1)
227 227 
       coderay (~> 1.1.0)
228 228 
       method_source (~> 0.8.1)
229 229 
       slop (~> 3.4)
 
@@ -232,24 +232,24 @@ GEM
232 232 
     rack (1.5.2)
233 233 
     rack-test (0.6.2)
234 234 
       rack (>= 1.0)
235 
-    rails (4.1.4)
236 
-      actionmailer (= 4.1.4)
237 
-      actionpack (= 4.1.4)
238 
-      actionview (= 4.1.4)
239 
-      activemodel (= 4.1.4)
240 
-      activerecord (= 4.1.4)
241 
-      activesupport (= 4.1.4)
235 
+    rails (4.1.5)
236 
+      actionmailer (= 4.1.5)
237 
+      actionpack (= 4.1.5)
238 
+      actionview (= 4.1.5)
239 
+      activemodel (= 4.1.5)
240 
+      activerecord (= 4.1.5)
241 
+      activesupport (= 4.1.5)
242 242 
       bundler (>= 1.3.0, < 2.0)
243 
-      railties (= 4.1.4)
243 
+      railties (= 4.1.5)
244 244 
       sprockets-rails (~> 2.0)
245 245 
     rails_12factor (0.0.2)
246 246 
       rails_serve_static_assets
247 247 
       rails_stdout_logging
248 248 
     rails_serve_static_assets (0.0.2)
249 249 
     rails_stdout_logging (0.0.3)
250 
-    railties (4.1.4)
251 
-      actionpack (= 4.1.4)
252 
-      activesupport (= 4.1.4)
250 
+    railties (4.1.5)
251 
+      actionpack (= 4.1.5)
252 
+      activesupport (= 4.1.5)
253 253 
       rake (>= 0.8.7)
254 254 
       thor (>= 0.18.1, < 2.0)
255 255 
     raindrops (0.13.0)
 
@@ -268,7 +268,7 @@ GEM
268 268 
       rspec-mocks (~> 2.99.0)
269 269 
     rspec-collection_matchers (1.0.0)
270 270 
       rspec-expectations (>= 2.99.0.beta1)
271 
-    rspec-core (2.99.1)
271 
+    rspec-core (2.99.2)
272 272 
     rspec-expectations (2.99.2)
273 273 
       diff-lcs (>= 1.1.3, < 2.0)
274 274 
     rspec-mocks (2.99.2)
 
@@ -296,7 +296,7 @@ GEM
296 296 
       sass (~> 3.2.0)
297 297 
       sprockets (~> 2.8, <= 2.11.0)
298 298 
       sprockets-rails (~> 2.0)
299 
-    select2-rails (3.5.9)
299 
+    select2-rails (3.5.9.1)
300 300 
       thor (~> 0.14)
301 301 
     shoulda-matchers (2.6.2)
302 302 
       activesupport (>= 3.0.0)
 
@@ -332,7 +332,7 @@ GEM
332 332 
     thor (0.19.1)
333 333 
     thread_safe (0.3.4)
334 334 
     tilt (1.4.1)
335 
-    tins (1.3.0)
335 
+    tins (1.3.2)
336 336 
     treetop (1.4.15)
337 337 
       polyglot
338 338 
       polyglot (>= 0.3.1)
 
@@ -353,7 +353,7 @@ GEM
353 353 
       simple_oauth (~> 0.2.0)
354 354 
     typhoeus (0.6.9)
355 355 
       ethon (>= 0.7.1)
356 
-    tzinfo (1.2.1)
356 
+    tzinfo (1.2.2)
357 357 
       thread_safe (~> 0.1)
358 358 
     uglifier (2.5.3)
359 359 
       execjs (>= 0.3.0)
 
@@ -364,7 +364,7 @@ GEM
364 364 
       raindrops (~> 0.7)
365 365 
     uuid (2.3.7)
366 366 
       macaddr (~> 1.0)
367 
-    uuidtools (2.1.4)
367 
+    uuidtools (2.1.5)
368 368 
     vcr (2.9.2)
369 369 
     warden (1.2.3)
370 370 
       rack (>= 1.0)
 
@@ -430,7 +430,7 @@ DEPENDENCIES
430 430 
   pry
431 431 
   quiet_assets
432 432 
   rack
433 
-  rails (= 4.1.4)
433 
+  rails (= 4.1.5)
434 434 
   rails_12factor
435 435 
   rr
436 436 
   rspec (~> 2.99)

+ 4 - 0
app/assets/javascripts/application.js.coffee.erb
View File 

@@ -60,6 +60,10 @@ showEventDescriptions = ->
60 60 
     $(".event-descriptions").html("").hide()
61 61
62 62 
 $(document).ready ->
63 
+  $('.navbar .dropdown.dropdown-hover').hover \
64 
+    -> $(this).addClass('open'),
65 
+    -> $(this).removeClass('open')
66 
+
63 67 
   # JSON Editor
64 68 
   window.jsonEditor = setupJsonEditor()[0]
65 69

+ 6 - 0
app/assets/stylesheets/application.css.scss.erb
View File 

@@ -98,6 +98,12 @@ span.not-applicable:after {
98 98 
   right: 1px;
99 99 
 }
100 100
101 
+.navbar {
102 
+  .dropdown.dropdown-hover:hover .dropdown-menu {
103 
+    display: block;
104 
+  }
105 
+}
106 
+
101 107 
 // Flash
102 108
103 109 
 .flash {

+ 15 - 4
app/concerns/web_request_concern.rb
View File 

@@ -1,3 +1,6 @@
1 
+require 'faraday'
2 
+require 'faraday_middleware'
3 
+
1 4 
 module WebRequestConcern
2 5 
   extend ActiveSupport::Concern
3 6 
 
@@ -21,9 +24,7 @@ module WebRequestConcern
21 24 
     @faraday ||= Faraday.new { |builder|
22 25 
       builder.headers = headers if headers.length > 0
23 26
24 
-      if (user_agent = interpolated['user_agent']).present?
25 
-        builder.headers[:user_agent] = user_agent
26 
-      end
27 
+      builder.headers[:user_agent] = user_agent
27 28
28 29 
       builder.use FaradayMiddleware::FollowRedirects
29 30 
       builder.request :url_encoded
 
@@ -58,4 +59,14 @@ module WebRequestConcern
58 59 
   def faraday_backend
59 60 
     ENV.fetch('FARADAY_HTTP_BACKEND', 'typhoeus').to_sym
60 61 
   end
61 
-end
62 
+
63 
+  def user_agent
64 
+    interpolated['user_agent'].presence || self.class.default_user_agent
65 
+  end
66 
+
67 
+  module ClassMethods
68 
+    def default_user_agent
69 
+      ENV.fetch('DEFAULT_HTTP_USER_AGENT', Faraday.new.headers[:user_agent])
70 
+    end
71 
+  end
72 
+end

+ 28 - 14
app/helpers/application_helper.rb
View File 

@@ -1,27 +1,41 @@
1 1 
 module ApplicationHelper
2 
-  def nav_link(name, path, options = {})
3 
-    (<<-HTML).html_safe
4 
-      <li class='#{(current_page?(path) ? "active" : "")}'>
5 
-        #{link_to name, path}
6 
-      </li>
7 
-    HTML
2 
+  def nav_link(name, path, options = {}, &block)
3 
+    if glyphicon = options.delete(:glyphicon)
4 
+      name = "<span class='glyphicon glyphicon-#{glyphicon}'></span> ".html_safe + name
5 
+    end
6 
+    content = link_to(name, path, options)
7 
+    active = current_page?(path)
8 
+    if block
9 
+      # Passing a block signifies that the link is a header of a hover
10 
+      # menu which contains what's in the block.
11 
+      begin
12 
+        @nav_in_menu = true
13 
+        @nav_link_active = active
14 
+        content += capture(&block)
15 
+        class_name = "dropdown dropdown-hover #{@nav_link_active ? 'active' : ''}"
16 
+      ensure
17 
+        @nav_in_menu = @nav_link_active = false
18 
+      end
19 
+    else
20 
+      # Mark the menu header active if it contains the current page
21 
+      @nav_link_active ||= active if @nav_in_menu
22 
+      # An "active" menu item may be an eyesore, hence `!@nav_in_menu &&`.
23 
+      class_name = !@nav_in_menu && active ? 'active' : ''
24 
+    end
25 
+    content_tag :li, content, class: class_name
8 26 
   end
9 27
10 28 
   def yes_no(bool)
11 
-    if bool
12 
-      '<span class="label label-info">Yes</span>'.html_safe
13 
-    else
14 
-      '<span class="label label-default">No</span>'.html_safe
15 
-    end
29 
+    content_tag :span, bool ? 'Yes' : 'No', class: "label #{bool ? 'label-info' : 'label-default' }"
16 30 
   end
17 31
18 32 
   def working(agent)
19 33 
     if agent.disabled?
20 
-      link_to 'Disabled', agent_path(agent), :class => 'label label-warning'
34 
+      link_to 'Disabled', agent_path(agent), class: 'label label-warning'
21 35 
     elsif agent.working?
22 
-      '<span class="label label-success">Yes</span>'.html_safe
36 
+      content_tag :span, 'Yes', class: 'label label-success'
23 37 
     else
24 
-      link_to 'No', agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details')), :class => 'label label-danger'
38 
+      link_to 'No', agent_path(agent, tab: (agent.recent_error_logs? ? 'logs' : 'details')), class: 'label label-danger'
25 39 
     end
26 40 
   end
27 41 
 end

+ 7 - 0
app/helpers/markdown_helper.rb
View File 

@@ -0,0 +1,7 @@
1 
+module MarkdownHelper
2 
+
3 
+  def markdown(text)
4 
+    Kramdown::Document.new(text, :auto_ids => false).to_html.html_safe
5 
+  end
6 
+
7 
+end

+ 2 - 0
app/models/agent.rb
View File 

@@ -56,6 +56,8 @@ class Agent < ActiveRecord::Base
56 56 
   has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :agent
57 57 
   has_many :scenarios, :through => :scenario_memberships, :inverse_of => :agents
58 58
59 
+  scope :active, -> { where(disabled: false) }
60 
+
59 61 
   scope :of_type, lambda { |type|
60 62 
     type = case type
61 63 
              when String, Symbol, Class

+ 1 - 1
app/models/agents/hipchat_agent.rb
View File 

@@ -43,7 +43,7 @@ module Agents
43 43 
       client = HipChat::Client.new(interpolated[:auth_token] || credential('hipchat_auth_token'))
44 44 
       incoming_events.each do |event|
45 45 
         mo = interpolated(event)
46 
-        client[mo[:room_name]].send(mo[:username], mo[:message], :notify => boolify(mo[:notify]) ? 1 : 0, :color => mo[:color])
46 
+        client[mo[:room_name]].send(mo[:username], mo[:message], :notify => boolify(mo[:notify]), :color => mo[:color])
47 47 
       end
48 48 
     end
49 49 
   end

+ 40 - 7
app/models/agents/website_agent.rb
View File 

@@ -1,6 +1,4 @@
1 1 
 require 'nokogiri'
2 
-require 'faraday'
3 
-require 'faraday_middleware'
4 2 
 require 'date'
5 3
6 4 
 module Agents
 
@@ -19,7 +17,7 @@ module Agents
19 17
20 18 
       `url` can be a single url, or an array of urls (for example, for multiple pages with the exact same structure but different content to scrape)
21 19
22 
-      The `type` value can be `xml`, `html`, or `json`.
20 
+      The `type` value can be `xml`, `html`, `json`, or `text`.
23 21
24 22 
       To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
25 23 
 
@@ -40,6 +38,28 @@ module Agents
40 38 
             "description": { "path": "results.data[*].description" }
41 39 
           }
42 40
41 
+      When parsing text, each sub-hash should contain a `regexp` and `index`.  Output text is matched against the regular expression repeatedly from the beginning through to the end, collecting a captured group specified by `index` in each match.  Each index should be either an integer or a string name which corresponds to `(?<_name_>...)`.  For example, to parse lines of `_word_: _definition_`, the following should work:
42 
+
43 
+          "extract": {
44 
+            "word": { "regexp": "^(.+?): (.+)$", index: 1 },
45 
+            "definition": { "regexp": "^(.+?): (.+)$", index: 2 },
46 
+          }
47 
+
48 
+      Or if you prefer names to numbers for index:
49 
+
50 
+          "extract": {
51 
+            "word": { "regexp": "^(?<word>.+?): (?<definition>.+)$", index: 'word' },
52 
+            "definition": { "regexp": "^(?<word>.+?): (?<definition>.+)$", index: 'definition' },
53 
+          }
54 
+
55 
+      To extract the whole content as one event:
56 
+
57 
+          "extract": {
58 
+            "content": { "regexp": "\A(?m:.)*\z", index: 0 },
59 
+          }
60 
+
61 
+      Beware that `.` does not match the newline character (LF) unless the `m` flag is in effect, and `^`/`$` basically match every line beginning/end.  See [this document](http://ruby-doc.org/core-#{RUBY_VERSION}/doc/regexp_rdoc.html) to learn the regular expression variant used in this service.
62 
+
43 63 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
44 64
45 65 
       Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`.
 
@@ -50,7 +70,7 @@ module Agents
50 70
51 71 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
52 72
53 
-      Set `user_agent` to a custom User-Agent name if the website does not like the default value ("Faraday v#{Faraday::VERSION}").
73 
+      Set `user_agent` to a custom User-Agent name if the website does not like the default value (`#{default_user_agent}`).
54 74
55 75 
       The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
56 76 
 
@@ -140,7 +160,15 @@ module Agents
140 160 
           else
141 161 
             output = {}
142 162 
             interpolated['extract'].each do |name, extraction_details|
143 
-              if extraction_type == "json"
163 
+              case extraction_type
164 
+              when "text"
165 
+                regexp = Regexp.new(extraction_details['regexp'])
166 
+                result = []
167 
+                doc.scan(regexp) {
168 
+                  result << Regexp.last_match[extraction_details['index']]
169 
+                }
170 
+                log "Extracting #{extraction_type} at #{regexp}: #{result}"
171 
+              when "json"
144 172 
                 result = Utils.values_at(doc, extraction_details['path'])
145 173 
                 log "Extracting #{extraction_type} at #{extraction_details['path']}: #{result}"
146 174 
               else
 
@@ -253,10 +281,13 @@ module Agents
253 281
254 282 
     def extraction_type
255 283 
       (interpolated['type'] || begin
256 
-        if interpolated['url'] =~ /\.(rss|xml)$/i
284 
+        case interpolated['url']
285 
+        when /\.(rss|xml)$/i
257 286 
           "xml"
258 
-        elsif interpolated['url'] =~ /\.json$/i
287 
+        when /\.json$/i
259 288 
           "json"
289 
+        when /\.(txt|text)$/i
290 
+          "text"
260 291 
         else
261 292 
           "html"
262 293 
         end
 
@@ -271,6 +302,8 @@ module Agents
271 302 
           JSON.parse(data)
272 303 
         when "html"
273 304 
           Nokogiri::HTML(data)
305 
+        when "text"
306 
+          data
274 307 
         else
275 308 
           raise "Unknown extraction type #{extraction_type}"
276 309 
       end

+ 4 - 4
app/views/agents/show.html.erb
View File 

@@ -110,8 +110,8 @@
110 110 
             <% if @agent.can_receive_events? %>
111 111 
               <p>
112 112 
                 <b>Event sources:</b>
113 
-                <% if @agent.sources.length %>
114 
-                  <%= @agent.sources.map { |source_agent| link_to(source_agent.name, agent_path(source_agent)) }.to_sentence.html_safe %>
113 
+                <% if (agents = @agent.sources).length > 0 %>
114 
+                  <%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %>
115 115 
                 <% else %>
116 116 
                   None
117 117 
                 <% end %>
 
@@ -126,8 +126,8 @@
126 126 
             <% if @agent.can_create_events? %>
127 127 
               <p>
128 128 
                 <b>Event receivers:</b>
129 
-                <% if @agent.receivers.length %>
130 
-                  <%= @agent.receivers.map { |receiver_agent| link_to(receiver_agent.name, agent_path(receiver_agent)) }.to_sentence.html_safe %>
129 
+                <% if (agents = @agent.receivers).length > 0 %>
130 
+                  <%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %>
131 131 
                 <% else %>
132 132 
                   None
133 133 
                 <% end %>

+ 7 - 1
app/views/layouts/_navigation.html.erb
View File 

@@ -12,7 +12,13 @@
12 12
13 13 
   <% if user_signed_in? %>
14 14 
     <ul class='nav navbar-nav'>
15 
-      <%= nav_link "Agents", agents_path %>
15 
+      <%= nav_link "Agents", agents_path do %>
16 
+        <ul class='dropdown-menu' role='menu'>
17 
+          <%= nav_link "New Agent", new_agent_path, glyphicon: "plus" %>
18 
+          <%= nav_link "Run event propagation", propagate_agents_path, method: 'post', glyphicon: "refresh" %>
19 
+          <%= nav_link "View Diagram", diagram_path, glyphicon: 'random' %>
20 
+        </ul>
21 
+      <% end %>
16 22 
       <%= nav_link "Scenarios", scenarios_path %>
17 23 
       <%= nav_link "Events", events_path %>
18 24 
       <%= nav_link "Credentials", user_credentials_path %>

+ 1 - 1
app/views/scenario_imports/_step_two.html.erb
View File 

@@ -30,7 +30,7 @@
30 30 
     </div>
31 31
32 32 
     <% if @scenario_import.parsed_data["description"].present? %>
33 
-      <blockquote><%= @scenario_import.parsed_data["description"] %></blockquote>
33 
+      <blockquote><%= markdown(@scenario_import.parsed_data["description"]) %></blockquote>
34 34 
     <% end %>
35 35
36 36 
   </div>

+ 1 - 1
app/views/scenarios/show.html.erb
View File 

@@ -6,7 +6,7 @@
6 6 
       </div>
7 7
8 8 
       <% if @scenario.description.present? %>
9 
-        <blockquote><%= @scenario.description %></blockquote>
9 
+        <blockquote><%= markdown(@scenario.description) %></blockquote>
10 10 
       <% end %>
11 11
12 12 
       <%= render 'agents/table', :returnTo => scenario_path(@scenario) %>

+ 8 - 8
lib/twitter_stream.rb
View File 

@@ -11,7 +11,6 @@ class TwitterStream
11 11
12 12 
   def stop
13 13 
     @running = false
14 
-    EventMachine::stop_event_loop if EventMachine.reactor_running?
15 14 
   end
16 15
17 16 
   def stream!(filters, agent, &block)
 
@@ -91,9 +90,13 @@ class TwitterStream
91 90 
   def run
92 91 
     while @running
93 92 
       begin
94 
-        agents = Agents::TwitterStreamAgent.all
93 
+        agents = Agents::TwitterStreamAgent.active.all
95 94
96 95 
         EventMachine::run do
96 
+          EventMachine.add_periodic_timer(1) {
97 
+            EventMachine::stop_event_loop if !@running
98 
+          }
99 
+
97 100 
           EventMachine.add_periodic_timer(RELOAD_TIMEOUT) {
98 101 
             puts "Reloading EventMachine and all Agents..."
99 102 
             EventMachine::stop_event_loop
 
@@ -101,17 +104,14 @@ class TwitterStream
101 104
102 105 
           if agents.length == 0
103 106 
             puts "No agents found.  Will look again in a minute."
104 
-            sleep 60
105 
-            EventMachine::stop_event_loop
107 
+            EventMachine.add_timer(60) {
108 
+              EventMachine::stop_event_loop
109 
+            }
106 110 
           else
107 111 
             puts "Found #{agents.length} agent(s).  Loading them now..."
108 112 
             load_and_run agents
109 113 
           end
110 114 
         end
111 
-
112 
-        print "Pausing..."; STDOUT.flush
113 
-        sleep 1
114 
-        puts "done."
115 115 
       rescue SignalException, SystemExit
116 116 
         @running = false
117 117 
         EventMachine::stop_event_loop if EventMachine.reactor_running?

+ 14 - 0
spec/helpers/markdown_helper_spec.rb
View File 

@@ -0,0 +1,14 @@
1 
+require 'spec_helper'
2 
+
3 
+describe MarkdownHelper do
4 
+
5 
+  describe '#markdown' do
6 
+
7 
+    it 'renders HTML from a markdown text' do
8 
+      markdown('# Header').should =~ /<h1>Header<\/h1>/
9 
+      markdown('## Header 2').should =~ /<h2>Header 2<\/h2>/
10 
+    end
11 
+
12 
+  end
13 
+
14 
+end

+ 2 - 2
spec/models/agents/hipchat_agent_spec.rb
View File 

@@ -7,7 +7,7 @@ describe Agents::HipchatAgent do
7 7 
                       'room_name' => 'test',
8 8 
                       'username' => "{{username}}",
9 9 
                       'message' => "{{message}}",
10 
-                      'notify' => false,
10 
+                      'notify' => 'false',
11 11 
                       'color' => 'yellow',
12 12 
                     }
13 13 
 
@@ -53,7 +53,7 @@ describe Agents::HipchatAgent do
53 53 
   describe "#receive" do
54 54 
     it "send a message to the hipchat" do
55 55 
       any_instance_of(HipChat::Room) do |obj|
56 
-        mock(obj).send(@event.payload[:username], @event.payload[:message], {:notify => 0, :color => 'yellow'})
56 
+        mock(obj).send(@event.payload[:username], @event.payload[:message], {:notify => false, :color => 'yellow'})
57 57 
       end
58 58 
       @checker.receive([@event])
59 59 
     end

+ 52 - 0
spec/models/agents/website_agent_spec.rb
View File 

@@ -398,6 +398,58 @@ describe Agents::WebsiteAgent do
398 398 
           event.payload['response']['title'].should == "hello!"
399 399 
         end
400 400 
       end
401 
+
402 
+      describe "text parsing" do
403 
+        before do
404 
+          stub_request(:any, /text-site/).to_return(body: <<-EOF, status: 200)
405 
+water: wet
406 
+fire: hot
407 
+          EOF
408 
+          site = {
409 
+            'name' => 'Some Text Response',
410 
+            'expected_update_period_in_days' => '2',
411 
+            'type' => 'text',
412 
+            'url' => 'http://text-site.com',
413 
+            'mode' => 'on_change',
414 
+            'extract' => {
415 
+              'word' => { 'regexp' => '^(.+?): (.+)$', index: 1 },
416 
+              'property' => { 'regexp' => '^(.+?): (.+)$', index: 2 },
417 
+            }
418 
+          }
419 
+          @checker = Agents::WebsiteAgent.new(name: 'Text Site', options: site)
420 
+          @checker.user = users(:bob)
421 
+          @checker.save!
422 
+        end
423 
+
424 
+        it "works with regexp" do
425 
+          @checker.options = @checker.options.merge('extract' => {
426 
+            'word' => { 'regexp' => '^(?<word>.+?): (?<property>.+)$', index: 'word' },
427 
+            'property' => { 'regexp' => '^(?<word>.+?): (?<property>.+)$', index: 'property' },
428 
+          })
429 
+
430 
+          lambda {
431 
+            @checker.check
432 
+          }.should change { Event.count }.by(2)
433 
+
434 
+          event1, event2 = Event.last(2)
435 
+          event1.payload['word'].should == 'water'
436 
+          event1.payload['property'].should == 'wet'
437 
+          event2.payload['word'].should == 'fire'
438 
+          event2.payload['property'].should == 'hot'
439 
+        end
440 
+
441 
+        it "works with regexp with named capture" do
442 
+          lambda {
443 
+            @checker.check
444 
+          }.should change { Event.count }.by(2)
445 
+
446 
+          event1, event2 = Event.last(2)
447 
+          event1.payload['word'].should == 'water'
448 
+          event1.payload['property'].should == 'wet'
449 
+          event2.payload['word'].should == 'fire'
450 
+          event2.payload['property'].should == 'hot'
451 
+        end
452 
+      end
401 453 
     end
402 454
403 455 
     describe "#receive" do

+ 26 - 1
spec/support/shared_examples/web_request_concern.rb
View File 

@@ -63,4 +63,29 @@ shared_examples_for WebRequestConcern do
63 63 
       agent.should_not be_valid
64 64 
     end
65 65 
   end
66 
-end
66 
+
67 
+  describe "User-Agent" do
68 
+    before do
69 
+      @default_http_user_agent = ENV['DEFAULT_HTTP_USER_AGENT']
70 
+      ENV['DEFAULT_HTTP_USER_AGENT'] = nil
71 
+    end
72 
+
73 
+    after do
74 
+      ENV['DEFAULT_HTTP_USER_AGENT'] = @default_http_user_agent
75 
+    end
76 
+
77 
+    it "should have the default value set by Faraday" do
78 
+      agent.user_agent.should == Faraday.new.headers[:user_agent]
79 
+    end
80 
+
81 
+    it "should be overridden by the environment variable if present" do
82 
+      ENV['DEFAULT_HTTP_USER_AGENT'] = 'Huginn - https://github.com/cantino/huginn'
83 
+      agent.user_agent.should == 'Huginn - https://github.com/cantino/huginn'
84 
+    end
85 
+
86 
+    it "should be overriden by the value in options if present" do
87 
+      agent.options['user_agent'] = 'Override'
88 
+      agent.user_agent.should == 'Override'
89 
+    end
90 
+  end
91 
+end